import samplerate
from tensorflow import keras
from keras import losses
from keras.callbacks import EarlyStopping
from keras.optimizers import adam
import os
import sys
os.environ["CUDA_VISIBLE_DEVICES"]="4"
sys.path.insert(1,'/home/jgozlan/GIT/scripts/')
sys.path.insert(1,'/home/jgozlan/GIT/models/')
sys.path.insert(1, '/home/jgozlan/GIT/models_analysis/')
import tensorflow as tf
tf.test.is_gpu_available()
import matplotlib.pyplot as plt
from imu_extractor import get_all_dataframes_for_forecasting, get_freq_magn, transform_dataset_into_freq_magn
from train_test_split import get_train_test_split
from baseline import find_frequency,repeat_period_forecast_acf, get_mean_line_forecast, get_flat_line_forecast
from metrics import compare_method, plot_few_prediction
from models import create_lstm_encoder_decoder
path_cluster0 = ['Run_Features']
path_cluster1 = ['CarDriving_Features']
path_cluster2 = ['Karting_Features',
'MotorcycleHelmet_Features',
'Scooter_Features','SkateboardChesty_Features',
'SnowboardSeeker_Features','Hiking_Features']
path_all_classes = path_cluster0 + path_cluster1 + path_cluster2
recording_freq = 6400.0
downsampling_freq = 200.0
all_dfs_classes = []
for i,c in enumerate(path_cluster0):
dfs = get_all_dataframes_for_forecasting("Database/" + c, recording_freq, downsampling_freq)
all_dfs_classes.append(dfs)
all_df = [item for sublist in all_dfs_classes for item in sublist]
lag = 500
ahead = 100
delay = 1
test_size = 0.2
dim = 12
target_index = [0,1,2]
classification = False
X_train, y_train, X_test, y_test = get_train_test_split(all_df, test_size, lag, ahead, delay, target_index, classification = False)
num_freq = 12
transformed_X_train = transform_dataset_into_freq_magn(X_train, downsampling_freq, num_freq)
transformed_X_test = transform_dataset_into_freq_magn(X_test, downsampling_freq, num_freq)
freq_dim_in = transformed_X_train.shape[1]
freq_dim_in
import pickle
from numpy import unique
from numpy import where
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
import matplotlib.pyplot as plt
import pickle
cluster_classification_model = keras.models.load_model('models_analysis/cluster4_classification_model_new')
running_forecasting_model = keras.models.load_model('models_analysis/running_forecasting_multi_step1')
lda_model = pickle.load(open('models_analysis/lda_model_time.sav', 'rb'))
kmeans_model = pickle.load(open('models_analysis/kmeans_model_time.sav', 'rb'))
gmm_model = pickle.load(open('models_analysis/gmm_model.sav', 'rb'))
X_train.shape, X_test.shape, transformed_X_train.shape, transformed_X_test.shape
forecast = running_forecasting_model.predict(X_test)
forecast_x = forecast[:,:,0].reshape((forecast.shape[0],forecast.shape[1]))
forecast_y = forecast[:,:,1].reshape((forecast.shape[0],forecast.shape[1]))
forecast_z = forecast[:,:,2].reshape((forecast.shape[0],forecast.shape[1]))
import random
import numpy as np
from evaluation2 import plot_forecasts2, plot_metrics, get_metrics_for_forecast
import matplotlib.pyplot as plt
mae_threshold_x_l = np.zeros((100,ahead))
mae_threshold_y_l = np.zeros((100,ahead))
mae_threshold_z_l = np.zeros((100,ahead))
mse_threshold_x_l = np.zeros((100,ahead))
mse_threshold_y_l = np.zeros((100,ahead))
mse_threshold_z_l = np.zeros((100,ahead))
mae_threshold_x_h = np.zeros((100,ahead))
mae_threshold_y_h = np.zeros((100,ahead))
mae_threshold_z_h = np.zeros((100,ahead))
mse_threshold_x_h = np.zeros((100,ahead))
mse_threshold_y_h = np.zeros((100,ahead))
mse_threshold_z_h = np.zeros((100,ahead))
mae_x_g = np.zeros((1,ahead))
mae_y_g = np.zeros((1,ahead))
mae_z_g = np.zeros((1,ahead))
mse_x_g = np.zeros((1,ahead))
mse_y_g = np.zeros((1,ahead))
mse_z_g = np.zeros((1,ahead))
cluster_probs = cluster_classification_model.predict(transformed_X_test)
cluster0_probs = cluster_probs[:,0]
mae_x, mse_x, mape_x, smape_x = get_metrics_for_forecast(y_test[:,:,0].reshape((-1,ahead)), forecast_x)
mae_y, mse_y, mape_y, smape_y = get_metrics_for_forecast(y_test[:,:,1].reshape((-1,ahead)), forecast_y)
mae_z, mse_z, mape_z, smape_z = get_metrics_for_forecast(y_test[:,:,2].reshape((-1,ahead)), forecast_z)
mae_x_g = mae_x
mae_y_g = mae_y
mae_z_g = mae_z
mse_x_g = mse_x
mse_y_g = mse_y
mse_z_g = mse_z
for threshold in range(0,100,1):
prob_threshold = threshold/100.0
idx_lower = np.where(cluster0_probs < prob_threshold)[0]
idx_higher = np.where(cluster0_probs > prob_threshold)[0]
print(prob_threshold, len(idx_lower), len(idx_higher))
if len(idx_lower) > 0:
mae_low_x, mse_low_x, mape_low_x, smape_low_x = get_metrics_for_forecast(y_test[idx_lower,:,0].reshape((-1,ahead)), forecast_x[idx_lower])
mae_low_y, mse_low_y, mape_low_y, smape_low_y = get_metrics_for_forecast(y_test[idx_lower,:,1].reshape((-1,ahead)), forecast_y[idx_lower])
mae_low_z, mse_low_z, mape_low_z, smape_low_z = get_metrics_for_forecast(y_test[idx_lower,:,2].reshape((-1,ahead)), forecast_z[idx_lower])
mae_threshold_x_l[threshold] = mae_low_x
mae_threshold_y_l[threshold] = mae_low_y
mae_threshold_z_l[threshold] = mae_low_z
mse_threshold_x_l[threshold] = mse_low_x
mse_threshold_y_l[threshold] = mse_low_y
mse_threshold_z_l[threshold] = mse_low_z
if len(idx_higher) > 0:
mae_high_x, mse_high_x, mape_high_x, smape_high_x = get_metrics_for_forecast(y_test[idx_higher,:,0].reshape((-1,ahead)), forecast_x[idx_higher])
mae_high_y, mse_high_y, mape_high_y, smape_high_y = get_metrics_for_forecast(y_test[idx_higher,:,1].reshape((-1,ahead)), forecast_y[idx_higher])
mae_high_z, mse_high_z, mape_high_z, smape_high_z = get_metrics_for_forecast(y_test[idx_higher,:,2].reshape((-1,ahead)), forecast_z[idx_higher])
mae_threshold_x_h[threshold] = mae_high_x
mae_threshold_y_h[threshold] = mae_high_y
mae_threshold_z_h[threshold] = mae_high_z
mse_threshold_x_h[threshold] = mse_high_x
mse_threshold_y_h[threshold] = mse_high_y
mse_threshold_z_h[threshold] = mse_high_z
def plot_metrics_for_threshold(metrics_threshold, metrics_global):
fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))
for i, ax in enumerate(axs.flatten()):
ax.plot(range(ahead), metrics_global[i], label = "global")
for threshold in range(90,100,1):
ax.plot(range(ahead), metrics_threshold[i][threshold], label = threshold)
ax.legend(loc="upper left")
plt.show()
plot_metrics_for_threshold([mae_threshold_x_l,mae_threshold_y_l,mae_threshold_z_l], [mae_x_g, mae_y_g, mae_z_g])
plot_metrics_for_threshold([mse_threshold_x_l,mse_threshold_y_l,mse_threshold_z_l],[mse_x_g, mse_y_g, mse_z_g])
plot_metrics_for_threshold([mae_threshold_x_h,mae_threshold_y_h,mae_threshold_z_h], [mae_x_g, mae_y_g, mae_z_g])
plot_metrics_for_threshold([mse_threshold_x_h,mse_threshold_y_h,mse_threshold_z_h], [mse_x_g, mse_y_g, mse_z_g])
import numpy as np
batch_pipeline_size = 100
threshold = 0.7
predictions_cluster = np.zeros((X_test.shape[0], ahead, 1 ))
predictions_global = np.zeros((X_test.shape[0], ahead, 1 ))
bad_indexes = np.zeros((X_test.shape[0]))
for i in range(0, len(X_test), batch_pipeline_size):
if (i % 5000) == 0:
print(i)
batch_time = X_test[i: (i+ batch_pipeline_size)]
batch_freq = transformed_X_test[i: (i+ batch_pipeline_size)]
cluster_probs = cluster_classification_model.predict(batch_freq)
cluster0_probs = cluster_probs[:,0]
idx = np.where(cluster0_probs < threshold)[0]
bad_indexes[i + idx] = 1
import random
from evaluation2 import plot_forecasts2, plot_metrics
import matplotlib.pyplot as plt
def plot_examples_test(indexes, X_true, Y_true, forecast, k):
random_indexes = random.sample(indexes, k)
for i in random_indexes:
fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))
x_input = X_true[i][:,0].reshape((lag,-1))
y_input = X_true[i][:,1].reshape((lag,-1))
z_input = X_true[i][:,2].reshape((lag,-1))
x_true = Y_true[i][:,0].reshape((ahead,-1))
y_true = Y_true[i][:,1].reshape((ahead,-1))
z_true = Y_true[i][:,2].reshape((ahead,-1))
x_forecast = forecast[i][:,0].reshape((ahead,-1))
y_forecast = forecast[i][:,1].reshape((ahead,-1))
z_forecast = forecast[i][:,2].reshape((ahead,-1))
t = [len(x_input) + j for j in range(len(x_true))]
#print(t)
inputs = [x_input, y_input,z_input]
outputs = [x_true , y_true , z_true]
forecasts = [x_forecast, y_forecast, z_forecast]
signal_labels = ['x_gyro signal', 'y_gyro_input signal','z_gyro signal']
signal_colors = ['red','green','blue']
forecast_labels = ['x_gyro forecast', 'y_gyro forecast','z_gyro forecast']
forecast_colors = ['orange','lime','cyan']
for i, ax in enumerate(axs.flatten()):
ax.plot(range(len(inputs[i])), inputs[i], label = signal_labels[i], color = signal_colors[i])
ax.plot(t, outputs[i], color = signal_colors[i])
ax.plot(t, forecasts[i], label = forecast_labels[i], color = forecast_colors[i])
ax.legend(loc="upper left")
plt.show()
k = 10
plot_examples_test(list(np.where(bad_indexes==0)[0]), X_test, y_test, forecast, k)
k = 10
plot_examples_test(list(np.where(bad_indexes==1)[0]), X_test, y_test, forecast, k)
transformed_X_test = X_test.reshape((-1,X_test.shape[1]*X_test.shape[2]))
X_random_lda3 = lda_model.transform(transformed_X_test)
from sklearn.metrics.pairwise import euclidean_distances
mae_threshold_x_l = np.zeros((100,ahead))
mae_threshold_y_l = np.zeros((100,ahead))
mae_threshold_z_l = np.zeros((100,ahead))
mse_threshold_x_l = np.zeros((100,ahead))
mse_threshold_y_l = np.zeros((100,ahead))
mse_threshold_z_l = np.zeros((100,ahead))
mae_threshold_x_h = np.zeros((100,ahead))
mae_threshold_y_h = np.zeros((100,ahead))
mae_threshold_z_h = np.zeros((100,ahead))
mse_threshold_x_h = np.zeros((100,ahead))
mse_threshold_y_h = np.zeros((100,ahead))
mse_threshold_z_h = np.zeros((100,ahead))
mae_x_g = np.zeros((1,ahead))
mae_y_g = np.zeros((1,ahead))
mae_z_g = np.zeros((1,ahead))
mse_x_g = np.zeros((1,ahead))
mse_y_g = np.zeros((1,ahead))
mse_z_g = np.zeros((1,ahead))
cluster_center = kmeans_model.cluster_centers_[-1]
distances_centroid_run = euclidean_distances(X_random_lda3, [cluster_center])
mae_x, mse_x, mape_x, smape_x = get_metrics_for_forecast(y_test[:,:,0].reshape((-1,ahead)), forecast_x)
mae_y, mse_y, mape_y, smape_y = get_metrics_for_forecast(y_test[:,:,1].reshape((-1,ahead)), forecast_y)
mae_z, mse_z, mape_z, smape_z = get_metrics_for_forecast(y_test[:,:,2].reshape((-1,ahead)), forecast_z)
mae_x_g = mae_x
mae_y_g = mae_y
mae_z_g = mae_z
mse_x_g = mse_x
mse_y_g = mse_y
mse_z_g = mse_z
for threshold in range(5,100,1):
current_threshold = threshold/10.0
idx_lower = np.where(distances_centroid_run < current_threshold)[0]
idx_higher = np.where(distances_centroid_run > current_threshold)[0]
#print(current_threshold, len(idx_lower), len(idx_higher))
if len(idx_lower) > 0:
mae_low_x, mse_low_x, mape_low_x, smape_low_x = get_metrics_for_forecast(y_test[idx_lower,:,0].reshape((-1,ahead)), forecast_x[idx_lower])
mae_low_y, mse_low_y, mape_low_y, smape_low_y = get_metrics_for_forecast(y_test[idx_lower,:,1].reshape((-1,ahead)), forecast_y[idx_lower])
mae_low_z, mse_low_z, mape_low_z, smape_low_z = get_metrics_for_forecast(y_test[idx_lower,:,2].reshape((-1,ahead)), forecast_z[idx_lower])
mae_threshold_x_l[threshold] = mae_low_x
mae_threshold_y_l[threshold] = mae_low_y
mae_threshold_z_l[threshold] = mae_low_z
mse_threshold_x_l[threshold] = mse_low_x
mse_threshold_y_l[threshold] = mse_low_y
mse_threshold_z_l[threshold] = mse_low_z
if len(idx_higher) > 0:
mae_high_x, mse_high_x, mape_high_x, smape_high_x = get_metrics_for_forecast(y_test[idx_higher,:,0].reshape((-1,ahead)), forecast_x[idx_higher])
mae_high_y, mse_high_y, mape_high_y, smape_high_y = get_metrics_for_forecast(y_test[idx_higher,:,1].reshape((-1,ahead)), forecast_y[idx_higher])
mae_high_z, mse_high_z, mape_high_z, smape_high_z = get_metrics_for_forecast(y_test[idx_higher,:,2].reshape((-1,ahead)), forecast_z[idx_higher])
mae_threshold_x_h[threshold] = mae_high_x
mae_threshold_y_h[threshold] = mae_high_y
mae_threshold_z_h[threshold] = mae_high_z
mse_threshold_x_h[threshold] = mse_high_x
mse_threshold_y_h[threshold] = mse_high_y
mse_threshold_z_h[threshold] = mse_high_z
print("THRESHOLD", current_threshold, len(idx_lower), len(idx_higher))
mae_x = [mae_threshold_x_l, mae_threshold_x_h]
mae_y = [mae_threshold_y_l, mae_threshold_y_h]
mae_z = [mae_threshold_z_l, mae_threshold_z_h]
fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))
for i, ax in enumerate(axs.flatten()):
if i == 0:
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[0], axis = 1), label = "X- under threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[1], axis = 1), label = "X- above threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_x_g), label = "X- global")
elif i == 1:
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[0], axis = 1), label = "Y- under threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[1], axis = 1), label = "Y- above threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_y_g), label = "Y - global")
else:
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[0], axis = 1), label = "Z- under threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[1], axis = 1), label = "Z- above threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_z_g), label = "Z - global")
ax.legend(loc="lower left")
plt.show()
def plot_metrics_for_euc_threshold(metrics_threshold, metrics_global):
fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))
for i, ax in enumerate(axs.flatten()):
ax.plot(range(ahead), metrics_global[i], label = "global", color='black')
for threshold in range(5,80,5):
ax.plot(range(ahead), metrics_threshold[i][threshold], label = threshold)
ax.legend(loc="upper left")
plt.show()
# LOWER THE BETTER
plot_metrics_for_euc_threshold([mae_threshold_x_l,mae_threshold_y_l,mae_threshold_z_l], [mae_x_g, mae_y_g, mae_z_g])
#HIGHER THE BETTER
plot_metrics_for_euc_threshold([mae_threshold_x_h,mae_threshold_y_h,mae_threshold_z_h], [mae_x_g, mae_y_g, mae_z_g])
threshold = 9.9
bad_indexes = np.zeros((X_test.shape[0]))
idx_lower = np.where(distances_centroid_run < threshold)[0]
idx_higher = np.where(distances_centroid_run > threshold)[0]
bad_indexes[idx_higher] = 1
k = 20
plot_examples_test(list(np.where(bad_indexes==1)[0]), X_test, y_test, forecast, k)
threshold = 8
bad_indexes = np.zeros((X_test.shape[0]))
idx_lower = np.where(distances_centroid_run < threshold)[0]
idx_higher = np.where(distances_centroid_run > threshold)[0]
bad_indexes[idx_higher] = 1
k = 20
plot_examples_test(list(np.where(bad_indexes==1)[0]), X_test, y_test, forecast, k)
# GMMMMMMMMMMMMMMMM
mae_threshold_x_l = np.zeros((100,ahead))
mae_threshold_y_l = np.zeros((100,ahead))
mae_threshold_z_l = np.zeros((100,ahead))
mse_threshold_x_l = np.zeros((100,ahead))
mse_threshold_y_l = np.zeros((100,ahead))
mse_threshold_z_l = np.zeros((100,ahead))
mae_threshold_x_h = np.zeros((100,ahead))
mae_threshold_y_h = np.zeros((100,ahead))
mae_threshold_z_h = np.zeros((100,ahead))
mse_threshold_x_h = np.zeros((100,ahead))
mse_threshold_y_h = np.zeros((100,ahead))
mse_threshold_z_h = np.zeros((100,ahead))
mae_x_g = np.zeros((1,ahead))
mae_y_g = np.zeros((1,ahead))
mae_z_g = np.zeros((1,ahead))
mse_x_g = np.zeros((1,ahead))
mse_y_g = np.zeros((1,ahead))
mse_z_g = np.zeros((1,ahead))
cluster_center = gmm_model.means_[0]
distances_centroid_run = euclidean_distances(X_random_lda3, [cluster_center])
mae_x, mse_x, mape_x, smape_x = get_metrics_for_forecast(y_test[:,:,0].reshape((-1,ahead)), forecast_x)
mae_y, mse_y, mape_y, smape_y = get_metrics_for_forecast(y_test[:,:,1].reshape((-1,ahead)), forecast_y)
mae_z, mse_z, mape_z, smape_z = get_metrics_for_forecast(y_test[:,:,2].reshape((-1,ahead)), forecast_z)
mae_x_g = mae_x
mae_y_g = mae_y
mae_z_g = mae_z
mse_x_g = mse_x
mse_y_g = mse_y
mse_z_g = mse_z
for threshold in range(0,100,1):
current_threshold = threshold/10.0
idx_lower = np.where(distances_centroid_run < current_threshold)[0]
idx_higher = np.where(distances_centroid_run > current_threshold)[0]
#print(current_threshold, len(idx_lower), len(idx_higher))
if len(idx_lower) > 0:
mae_low_x, mse_low_x, mape_low_x, smape_low_x = get_metrics_for_forecast(y_test[idx_lower,:,0].reshape((-1,ahead)), forecast_x[idx_lower])
mae_low_y, mse_low_y, mape_low_y, smape_low_y = get_metrics_for_forecast(y_test[idx_lower,:,1].reshape((-1,ahead)), forecast_y[idx_lower])
mae_low_z, mse_low_z, mape_low_z, smape_low_z = get_metrics_for_forecast(y_test[idx_lower,:,2].reshape((-1,ahead)), forecast_z[idx_lower])
mae_threshold_x_l[threshold] = mae_low_x
mae_threshold_y_l[threshold] = mae_low_y
mae_threshold_z_l[threshold] = mae_low_z
mse_threshold_x_l[threshold] = mse_low_x
mse_threshold_y_l[threshold] = mse_low_y
mse_threshold_z_l[threshold] = mse_low_z
if len(idx_higher) > 0:
mae_high_x, mse_high_x, mape_high_x, smape_high_x = get_metrics_for_forecast(y_test[idx_higher,:,0].reshape((-1,ahead)), forecast_x[idx_higher])
mae_high_y, mse_high_y, mape_high_y, smape_high_y = get_metrics_for_forecast(y_test[idx_higher,:,1].reshape((-1,ahead)), forecast_y[idx_higher])
mae_high_z, mse_high_z, mape_high_z, smape_high_z = get_metrics_for_forecast(y_test[idx_higher,:,2].reshape((-1,ahead)), forecast_z[idx_higher])
mae_threshold_x_h[threshold] = mae_high_x
mae_threshold_y_h[threshold] = mae_high_y
mae_threshold_z_h[threshold] = mae_high_z
mse_threshold_x_h[threshold] = mse_high_x
mse_threshold_y_h[threshold] = mse_high_y
mse_threshold_z_h[threshold] = mse_high_z
print("THRESHOLD", current_threshold, len(idx_lower), len(idx_higher))
mae_x = [mae_threshold_x_l, mae_threshold_x_h]
mae_y = [mae_threshold_y_l, mae_threshold_y_h]
mae_z = [mae_threshold_z_l, mae_threshold_z_h]
fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18,4))
for i, ax in enumerate(axs.flatten()):
if i == 0:
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[0], axis = 1), label = "X- under threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_x[1], axis = 1), label = "X- above threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_x_g), label = "X- global")
elif i == 1:
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[0], axis = 1), label = "Y- under threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_y[1], axis = 1), label = "Y- above threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_y_g), label = "Y - global")
else:
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[0], axis = 1), label = "Z- under threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.mean(mae_z[1], axis = 1), label = "Z- above threshold")
ax.plot(np.arange(0.0,10.0,0.1), np.ones((100))* np.mean(mae_z_g), label = "Z - global")
ax.legend(loc="lower left")
plt.show()